package io.github.biezhi.elves.spider;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;
import io.github.biezhi.elves.bidding.model.BiddingDTO;
import io.github.biezhi.elves.bidding.model.ChangeNoticeDTO;
import io.github.biezhi.elves.bidding.model.DownloadBiddingHtmlDO;
import io.github.biezhi.elves.bidding.model.DownloadBiddingHtmlDTO;
import io.github.biezhi.elves.bidding.model.DownloadFileDTO;
import io.github.biezhi.elves.bidding.model.NoticeTypeDTO;
import io.github.biezhi.elves.bidding.model.PurchaseModeDTO;
import io.github.biezhi.elves.bidding.model.RecordBiddingNoticeDTO;
import io.github.biezhi.elves.bidding.model.RecordNoticeQuery;
import io.github.biezhi.elves.bidding.model.WinBiddingDTO;
import io.github.biezhi.elves.common.BiddingNoticeOperationMode;
import io.github.biezhi.elves.common.Constants;
import io.github.biezhi.elves.common.FileType;
import io.github.biezhi.elves.model.BiddingQuery;
import io.github.biezhi.elves.model.NoticeTypeSelectedDTO;
import io.github.biezhi.elves.ok.http.FastHttpClient;
import io.github.biezhi.elves.service.BidService;
import io.github.biezhi.elves.utils.StringUtils;

/**
 *	投标数据管理模块组件
 *
 */
@Component
public class BidSpider {
	@Autowired
	private BidService bidService;
	
	private static final String indexUrl = "http://www.gdgpo.gov.cn";
	
	/**
	 * 开始执行爬取投标公告数据
	 * @throws Exception 
	 */
	//页码，默认第一页
	static int pageIndex = 1;
	//末页页码
	static int lastPageIndex = 0;
	//是否完成保存记录投标网公告标题数据
	static Boolean finished = false;
	public void startBid() throws Exception{
		biddingNoticeData(pageIndex);
	}
	
	/**
	 * 爬取投标公告数据
	 * @param pageNo		参数说明:页号
     * @param pageSize		参数说明:每页显示记录数
	 */
	Long sleepTime = 3000L;
	int indexI = 0;
	int indexJ = 0;
	static int listPurchaseModeIndex = 0;
	private void biddingNoticeData(int pageIndex) throws Exception{
		String keyword = "大数据";
		
		System.out.println("========开始请求=====");
		List<NoticeTypeDTO> noticeTypes = this.bidService.listNoticeType();
		for (int i = 0; i < noticeTypes.size(); i++) {
			NoticeTypeDTO noticeType = noticeTypes.get(i);
			saveBiddingAndDownloadHtml(noticeType, keyword);
		}
	}
	
	private void saveBiddingAndDownloadHtml(NoticeTypeDTO noticeType, String keyword) throws Exception {
		List<PurchaseModeDTO> purchaseModes = this.bidService.listPurchaseMode(noticeType.getNoticeId());
		System.out.println(purchaseModes+"xxxxxxxxxx");
		if (purchaseModes != null && purchaseModes.size() > 0) {
			for (int j = 0; j < purchaseModes.size(); j++) {
				PurchaseModeDTO purchaseMode = purchaseModes.get(j);
				NoticeTypeSelectedDTO noticeTypeSelected = new NoticeTypeSelectedDTO(purchaseMode.getPurchaseId(), noticeType.getNoticeName(), purchaseMode.getPurchaseName());
//				saveBiddingNotice(noticeTypeSelected, keyword, pageIndex, sleepTime);
				saveDownloadBiddingHtml(noticeTypeSelected, keyword);
			}
		} else {
			NoticeTypeSelectedDTO noticeTypeSelected = new NoticeTypeSelectedDTO(noticeType.getNoticeId(), noticeType.getNoticeName(), "");
//			saveBiddingNotice(noticeTypeSelected, keyword, pageIndex, sleepTime);
			saveDownloadBiddingHtml(noticeTypeSelected, keyword);
		}
//		biddingNoticeData(pageIndex);
	}
	
	/**
	 * 保存记录投标公告标题数据
	 * @param noticeTypeSelected
	 * @param keyword
	 * @param pageIndex
	 * @param sleepTime
	 * @throws Exception
	 */
	private void saveBiddingNotice(NoticeTypeSelectedDTO noticeTypeSelected, String keyword, Integer pageIndex, Long sleepTime) throws Exception {
//		if (finished && lastPageIndex > 0) {
//			biddingNoticeData(pageIndex);
//		}
		String result = postHtmlContent(keyword, noticeTypeSelected.getChannelCode(), pageIndex);
		Document doc = Jsoup.parse(result);
		Elements elements = doc.select(".m_m_cont > ul.m_m_c_list > li");
		//如果获取的数据列表为0，则休眠5000毫秒，重新调用biddingNoticeData方法
		if(elements == null || elements.size() == 0) {
			System.out.println("====开始休眠" + sleepTime + "毫秒====");
			sleepTime();
			System.out.println("====休眠" + sleepTime +"毫秒后，重新请求====");
			result = postHtmlContent(keyword, noticeTypeSelected.getChannelCode(), pageIndex);
		}
		if(elements!=null && elements.size()>0){
			for(Element listElement : elements){
				Elements rowElements = listElement.select("li");
				ListIterator<Element> resultIter = rowElements.listIterator();
				while(resultIter.hasNext()) {
					//创建记录投标公告标题数据对象
					RecordBiddingNoticeDTO recordBiddingNotice = null;
					try {
						recordBiddingNotice = createRecordBiddingNoticeDTO(resultIter, keyword, noticeTypeSelected, pageIndex);
					} catch (Exception e) {
						continue;
					}
					if(recordBiddingNotice == null) {
						continue;
					}
					this.bidService.saveRecordBiddingNotice(recordBiddingNotice);
	                //System.out.println(JSONObject.toJSON(recordBiddingNotice));
				}
			}
			
			//System.out.println("====已保存第" + pageIndex + "页数据=====");
			//System.out.println("====休眠" + sleepTime + "毫秒====");
			//sleepTime();
			//System.out.println("====休眠" + sleepTime +"毫秒后，往下执行====");
			
			//判断默认末页页码是否为0
			if(lastPageIndex > 0) {
				if(lastPageIndex > pageIndex) {
					//分页页码+1
					pageIndex++;
					//分页查询，递归调用
					saveBiddingNotice(noticeTypeSelected, keyword, pageIndex, sleepTime);
				} else {
					finished = true;
					//System.err.println("根据关键字：" + keyword + "，公告类型：" + noticeTypeSelected.getNoticeTypeName() + "，采购方式：" + noticeTypeSelected.getPurchaseModeName() + "查询条件，分页共" + lastPageIndex + "页，全部保存到t_record_bidding_notice表中");
					
					System.out.println();
					System.out.println("======开始下载公告详情内容页html数据=====");
					saveDownloadBiddingHtml(noticeTypeSelected, keyword);
				}
			} else {
				//默认末页页码为0，获取数据中的尾页页码
				
				//获取分页元素
				Elements pageElements = doc.select(".m_m_c_page > form");
				if(pageElements != null && pageElements.size() > 0){
					for(Element pageElement : pageElements){
						Elements pageAElements = pageElement.select("a");
						ListIterator<Element> pageResultIter = pageAElements.listIterator();
						while(pageResultIter.hasNext()) {
							Element get = pageResultIter.next();
						 	String pageText = get.text();
						 	pageText = pageText.replaceAll("\\s","");
						 	//获取尾页页码
						 	if(" 尾页 ".equals(pageText)) {
						 		String lastHref = get.attr("href");
						 		lastHref = lastHref.replace("javascript:turnOverPage(", "").replace(")", "");
						 		Integer lastHrefInt = Integer.parseInt(lastHref);
						 		//判断获取到尾页页码大于1，则存在下一页
						 		if(lastHrefInt > 1) {
						 			//将获取到尾页页码set到lastPageIndex中去
						 			lastPageIndex = lastHrefInt;
						 			System.out.println("总页数："+lastPageIndex);
						 			
						 			if(lastPageIndex > pageIndex) {
										//分页页码+1
										pageIndex++;
										//分页查询，递归调用
										saveBiddingNotice(noticeTypeSelected, keyword, pageIndex, sleepTime);
									}
						 		}
						 	}
						}
					}
				}
			}
		}
	}
	
	/**
	 * 保存公告正文内容数据到数据库
	 * @param noticeTypeSelected
	 * @param keyword
	 * @throws Exception
	 */
	private void saveDownloadBiddingHtml(NoticeTypeSelectedDTO noticeTypeSelected, String keyword) throws Exception {
		String noticeType = noticeTypeSelected.getNoticeTypeName();
        
		//查询未解析投标网公告标题数据集合
//		List<RecordBiddingNoticeDTO> listRecordBiddingNotice = new ArrayList<RecordBiddingNoticeDTO>();
//		RecordBiddingNoticeDTO recordBiddingNotice1 = new RecordBiddingNoticeDTO();
//		recordBiddingNotice1.setId("dddd");
//		recordBiddingNotice1.setNoticeType("采购公告");
//		recordBiddingNotice1.setPurchaseMethod("公开招标");
//		recordBiddingNotice1.setNoticeDetailUrl("http://www.gdgpo.gov.cn/showNotice/id/40288ba9641dd42a01641fde673005b6.html");
//		listRecordBiddingNotice.add(recordBiddingNotice1);
		
		List<RecordBiddingNoticeDTO> listRecordBiddingNotice = this.bidService.listRecordBiddingNotice(new RecordNoticeQuery(0, noticeType));
		if(listRecordBiddingNotice != null && listRecordBiddingNotice.size() > 0) {
			for(RecordBiddingNoticeDTO recordBiddingNotice: listRecordBiddingNotice) {
				BiddingQuery query = new BiddingQuery(recordBiddingNotice);
				Integer existBiddingCount = this.bidService.countDownloadBiddingHtmlByQuery(query);
				
				//如果大于0，则表示已经爬取了数据，不再爬取，继续循环
				if (existBiddingCount > 0) {
					System.out.println("downloadHtml已存在：" + query);
					continue;
				}
				
				String result = getHtmlContent(recordBiddingNotice.getNoticeDetailUrl());
				if (StringUtils.isNullOrEmpty(result)) {
					sleepTime();
					result = getHtmlContent(recordBiddingNotice.getNoticeDetailUrl());
				}
				Document doc = Jsoup.parse(result);
				Elements elements = doc.select(".zw_contianer");
				if (elements == null || elements.size() == 0) {
					sleepTime();
					result = getHtmlContent(recordBiddingNotice.getNoticeDetailUrl());
				}
				if(elements != null && elements.size() > 0) {
					DownloadBiddingHtmlDTO downloadBiddingHtml = new DownloadBiddingHtmlDTO();
					downloadBiddingHtml.setRecordId(recordBiddingNotice.getRecordId());
					downloadBiddingHtml.setNoticeType(recordBiddingNotice.getNoticeType());
					downloadBiddingHtml.setPurchaseMode(recordBiddingNotice.getPurchaseMode());
					downloadBiddingHtml.setKeyword(keyword);
					String html = result.replace("\r\n", "");
					downloadBiddingHtml.setHtmlContent(html);
					
					//公告详情页面正文内容
					Elements contElements = doc.select(".zw_c_c_cont");
					if(contElements != null && contElements.size() > 0) {
						//查找正文中所有a标签
						Elements allAElements = contElements.select("a");
						ListIterator<Element> htmlAResultIter = allAElements.listIterator();
						
						List<DownloadFileDTO> listDownloadFile = new ArrayList<DownloadFileDTO>();
						while(htmlAResultIter.hasNext()) {
							Element get = htmlAResultIter.next();
							String fileName = get.text();
							String fileUrl = indexUrl + get.attr("href");
							//文件后缀
							String[] fileSuffixs = FileType.FILE_SUFFIX.split(",");
							for(String fileSuffix : fileSuffixs) {
								//判断获取的文件链接与文件后缀是否匹配
								if(fileUrl.endsWith(fileSuffix)) {
									DownloadFileDTO downloadFile = new DownloadFileDTO();
									downloadFile.setFileName(fileName);
									downloadFile.setFileUrl(fileUrl);
									
									System.out.println("fileName：" + fileName);
									System.out.println("fileUrl：" + fileUrl);
									listDownloadFile.add(downloadFile);
									break;
								}
							}
							downloadBiddingHtml.setDownloadFile(JSONObject.toJSONString(listDownloadFile));
						}
					}
					//保存正文内容数据
					this.bidService.saveDownloadBiddingHtml(downloadBiddingHtml);
					this.bidService.updateRecordBiddingNotice("1", recordBiddingNotice.getId());
					System.out.println(JSONObject.toJSONString(downloadBiddingHtml));
				}
			}
			System.err.println("根据关键字：" + keyword + "，公告类型：" + noticeTypeSelected.getNoticeTypeName() + "，采购方式：" + noticeTypeSelected.getPurchaseModeName() + "查询条件，分页共" + lastPageIndex + "页，全部保存到t_download_bidding_html表中");
		}
	}
	
	/**
	 * 创建记录投标公告标题数据DTO对象
	 * @param resultIter				参数说明：迭代器
	 * @param keyword				参数说明：关键字
	 * @param channelCode		参数说明：关键字
	 * @param pageIndex			参数说明：分页页码
	 * @throws Exception
	 */
	private RecordBiddingNoticeDTO createRecordBiddingNoticeDTO(ListIterator<Element> resultIter, String keyword,
			NoticeTypeSelectedDTO noticeTypeSelected, Integer pageIndex) throws Exception {
		RecordBiddingNoticeDTO recordBiddingNotice = new RecordBiddingNoticeDTO();
	 	Element get = resultIter.next();

	 	String id = get.select("a").get(1).attr("href");
	 	//System.out.println(get.select("a"));
	 	//System.out.println(id);
        id = id.substring(id.lastIndexOf("/") + 1);
        id = id.replace(".html", "").replace(".htm", "");
        
        String title = get.select("a").attr("title");
        String pubdate = get.select("em").text();
        
        String noticeType = noticeTypeSelected.getNoticeTypeName();
        String purchaseMode = noticeTypeSelected.getPurchaseModeName();
        
        BiddingQuery query = new BiddingQuery(id, title, noticeType, purchaseMode, keyword);
        Integer exsitBiddingCount = this.bidService.countRecordBiddingNoticeById(query);
        if(exsitBiddingCount > 0) {
        	System.out.println("已存在数据：" + query);
        	return null;
        }
        
        recordBiddingNotice.setRecordId(id);
        recordBiddingNotice.setNoticeType(noticeType);
        recordBiddingNotice.setPurchaseMode(purchaseMode);
        recordBiddingNotice.setKeyword(keyword);
        recordBiddingNotice.setTitle(title);
        recordBiddingNotice.setNoticeDetailUrl(indexUrl + get.select("a").get(1).attr("href"));
        recordBiddingNotice.setSize(pageIndex);
        recordBiddingNotice.setOperationMode(BiddingNoticeOperationMode.ZERO);
        recordBiddingNotice.setPubdate(pubdate);
        return recordBiddingNotice;
	}
	
	/**
	 * 解析采购公告HTML
	 * @throws Exception 
	 */
	public void resolveBiddingHtml(String channelCode) throws Exception {
		String noticeType = "采购公告";
        if ("000801".equals(channelCode)) {
        	noticeType = "中标公告";
        }
        
		//查询未解析投标网公告标题数据集合
//		List<RecordBiddingNoticeDTO> listRecordBiddingNotice = new ArrayList<RecordBiddingNoticeDTO>();
//		RecordBiddingNoticeDTO recordBiddingNotice1 = new RecordBiddingNoticeDTO();
//		recordBiddingNotice1.setId("dddd");
//		recordBiddingNotice1.setNoticeType("采购公告");
//		recordBiddingNotice1.setPurchaseMode("公开招标");
//		recordBiddingNotice1.setNoticeDetailUrl("http://www.gdgpo.gov.cn/showNotice/id/40288ba963836745016386ff8df477f6.html");
//		recordBiddingNotice1.setNoticeDetailUrl("http://www.gdgpo.gov.cn/showNotice/id/40288ba96272641d016275c716152c17.html");
//		recordBiddingNotice1.setNoticeDetailUrl("http://www.gdgpo.gov.cn/showNotice/id/40288ba95f6dc535015f7b5763507863.html");
//		listRecordBiddingNotice.add(recordBiddingNotice1);
		
		List<RecordBiddingNoticeDTO> listRecordBiddingNotice = this.bidService.listRecordBiddingNotice(new RecordNoticeQuery(1, noticeType));
		if (listRecordBiddingNotice == null || listRecordBiddingNotice.size() == 0) {
			System.err.println("====没有要解析的数据");
			try {
				System.err.println("===开始休眠" + sleepTime + "毫秒......");
				Thread.sleep(sleepTime);
				System.err.println("===结束休眠，重新获取解析数据");
				resolveBiddingHtml(channelCode);
			} catch (Exception e) {}
		}
		if(listRecordBiddingNotice != null && listRecordBiddingNotice.size() > 0) {
			for(RecordBiddingNoticeDTO recordBiddingNotice: listRecordBiddingNotice) {
//				String result = getHtmlContent(recordBiddingNotice.getNoticeDetailUrl());
		
				DownloadBiddingHtmlDTO downloadBiddingHtmlByRecordId = this.bidService.downloadBiddingHtmlByRecordId(recordBiddingNotice.getRecordId());
				if(downloadBiddingHtmlByRecordId != null) {
					String result = downloadBiddingHtmlByRecordId.getHtmlContent();
					
					Document doc = Jsoup.parse(result);
					Elements elements = doc.select(".zw_contianer");
					if(elements != null && elements.size() > 0) {
						Boolean existSave = false;
						BiddingDTO bidding = new BiddingDTO();
						List<BiddingDTO> biddingList = new ArrayList<BiddingDTO>();
						
						String title = elements.select(".zw_c_cont > .zw_c_c_title").text();
						bidding.setProjectCode(StringUtils.matchProjectCode(title));
						
						//公告详情页面正文内容
						Elements contElements = doc.select(".zw_c_c_cont");
						if(contElements != null && contElements.size() > 0) {
							//查找正文中所有p标签下面的span标签
							Elements allPElements = contElements.select("p");
							if(allPElements != null && allPElements.size() > 0){
								for(Element allPElement : allPElements){
									Elements spanElements = allPElement.select("span");
									ListIterator<Element> htmlSpanResultIter = spanElements.listIterator();
									
									while(htmlSpanResultIter.hasNext()) {
										Element get = htmlSpanResultIter.next();
										String text = get.text();
										
										if(!text.contains("采购项目名称：") && (!text.contains("编号：") && !text.contains("项目编号："))
												&& (!text.contains("预算：") && !text.contains("预算金额（元）："))
												&& StringUtils.isNullOrEmpty(StringUtils.matchTargetContent(text))) {
											continue;
										}
										//获取发标人数据
										if(!StringUtils.isNullOrEmpty(StringUtils.matchTargetContent(text))) {
											bidding.setBidder(StringUtils.matchTargetContent(text));
										}
										
										if(text.contains("采购项目名称：")) {
											String projectName = "";
											if(text.split("：").length > 1) {
												projectName = text.split("：")[1];
											} else {
												System.out.println(spanElements.size());
												projectName = allPElement.select("span").get(1).text();
											}
											bidding.setProjectName(projectName.replace("\u00A0", "").replace("；", ""));
										}
										if(text.contains("编号：") || text.contains("项目编号：")) {
											try {
												String projectCode = "";
												if(text.split("：").length > 1) {
													projectCode = text.split("：")[1];
												} else {
													projectCode = allPElement.select("span").get(1).text();
												}
												//如果从正文title获取到项目编号，则从正文获取到编号，设置到代理项目编号
												if(!StringUtils.isNullOrEmpty(bidding.getProjectCode())) {
													bidding.setAgencyNumberCode(projectCode.trim().replace("\u00A0", "").replace("；", "").replace("　　　　　　　", ""));
												} else {
													bidding.setProjectCode(projectCode.trim().replace("\u00A0", "").replace("；", "").replace("　　　　　　　", ""));
												}
											}catch(Exception e) {
												System.err.println(text);
												e.printStackTrace();
											}
										}
										if(text.contains("预算：") || text.contains("预算金额（元）：")) {
											try {
												String biddingPrice = "";
												if(text.split("：").length > 1) {
													biddingPrice = text.split("：")[1];
												} else {
													String priceText = allPElement.select("span").get(1).text();
													if(!priceText.contains("万") && !priceText.contains("万元")) {
														biddingPrice = allPElement.select("span").get(1).text() + allPElement.select("span").get(2).text();
													} else {
														biddingPrice = allPElement.select("span").get(1).text();
													}
												}
												bidding.setBiddingPrice(biddingPrice.replace("\u00A0", "").replace("；", ""));
											}catch(Exception e) {
												System.err.println("projectName==>" + bidding.getProjectName() + "，获取到的文本==>" + text);
												e.printStackTrace();
											}
										}
	//									System.out.println(text);
										
										if(!StringUtils.isNullOrEmpty(bidding.getProjectName()) && !StringUtils.isNullOrEmpty(bidding.getProjectCode())
												&& !StringUtils.isNullOrEmpty(bidding.getBiddingPrice())) {
											System.out.println(JSONObject.toJSONString(bidding));
										} else {
											continue;
										}
										bidding.setRecordId(recordBiddingNotice.getRecordId());
										bidding.setPurchaseId("1");
										bidding.setNoticeId("2");
										
										//重新读取正文内容，获取发标时间
										spanElements = allPElements.select("p > font > span");
										if(spanElements != null && spanElements.size() > 0) {
											for(Element spanElement : spanElements) {
												text  = spanElement.text();
												if(StringUtils.isNullOrEmpty(StringUtils.regexDateByHtml(text)) && !text.contains("符合资格的供应商")) {
													continue;
												}
												//发标时间
												if(!StringUtils.isNullOrEmpty(StringUtils.regexDateByHtml(text)) || text.contains("符合资格的供应商")) {
													String markingTimeText = StringUtils.regexDateByHtml(text);
													if (StringUtils.isNullOrEmpty(markingTimeText)) {
														if (text.contains("符合资格的供应商")) {
															markingTimeText = spanElement.parent().text();
															markingTimeText = StringUtils.regexDateByHtml(markingTimeText);
														}
													}
													bidding.setMarkingTime(markingTimeText);
													biddingList.add(bidding);
													
													saveBiddingData(bidding, biddingList, recordBiddingNotice);
													
													bidding = new BiddingDTO();
													existSave = true;
													break;
												}
											}
										}
										
										if (!existSave) {
											spanElements = allPElements.select("p > span");
											htmlSpanResultIter = spanElements.listIterator();
											while(htmlSpanResultIter.hasNext()) {
												get = htmlSpanResultIter.next();
												text = get.text();
												
												if(StringUtils.isNullOrEmpty(StringUtils.regexDateByHtml(text))) {
//												if(StringUtils.isNullOrEmpty(StringUtils.regexDateByHtml(text)) && (!text.contains("开标时间：") && !text.contains("开标评标时间："))) {
													continue;
												}
												
												//发标时间
												if(!StringUtils.isNullOrEmpty(StringUtils.regexDateByHtml(text))) {
													bidding.setMarkingTime(StringUtils.regexDateByHtml(text));
													biddingList.add(bidding);
													
													saveBiddingData(bidding, biddingList, recordBiddingNotice);
													
													bidding = new BiddingDTO();
													break;
												}
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
	
	/**
	 * 保存发标人数据
	 * @param bidding
	 * @param biddingList
	 * @param recordBiddingNotice
	 */
	private void saveBiddingData(BiddingDTO bidding, List<BiddingDTO> biddingList, RecordBiddingNoticeDTO recordBiddingNotice) throws Exception {
		if(checkBiddingNotNull(bidding)) {
			if(biddingList != null && biddingList.size() > 0) {
				System.out.println();
				System.out.println("size：" + biddingList.size());
				System.out.println(JSONObject.toJSONString(biddingList));
				
				for(BiddingDTO biddingDTO : biddingList) {
					this.bidService.saveBidding(biddingDTO);
				}
			}
			
			this.bidService.updateRecordBiddingNotice("2", recordBiddingNotice.getId());
		}
	}
	
	/**
	 * 验证发标字段不为空
	 * @param bidding
	 * @return
	 */
	private Boolean checkBiddingNotNull(BiddingDTO bidding) {
		System.out.println("check==>" + JSONObject.toJSONString(bidding));
		if (!StringUtils.isNullOrEmpty(bidding.getBidder()) && !StringUtils.isNullOrEmpty(bidding.getProjectName())
				&& !StringUtils.isNullOrEmpty(bidding.getProjectCode()) && !StringUtils.isNullOrEmpty(bidding.getBiddingPrice())
				&& !StringUtils.isNullOrEmpty(bidding.getMarkingTime()) && !StringUtils.isNullOrEmpty(bidding.getPurchaseId())
				&& !StringUtils.isNullOrEmpty(bidding.getNoticeId())) {
			return true;
		}
		return false;
	}
	
	/**
	 * 解析中标公告HTML
	 * @throws Exception 
	 */
	public void resolveWinBiddingHtml() throws Exception {

		//根据t_download_bidding_html表，查询所有更正公告html
		//定义公告类型
		String noticeType = "中标公告";
		List<DownloadBiddingHtmlDTO> listDownloadBiddingHtmlDTO = bidService.downloadBiddingHtmlByNoticeType(noticeType);
		System.out.println(listDownloadBiddingHtmlDTO.size());
		if(listDownloadBiddingHtmlDTO != null && listDownloadBiddingHtmlDTO.size() > 0) {
			for (DownloadBiddingHtmlDTO downloadBiddingHtmlDTO : listDownloadBiddingHtmlDTO) {
				String result = downloadBiddingHtmlDTO.getHtmlContent();
				Document doc = Jsoup.parse(result);
				Elements elements = doc.select(".zw_contianer");
				if(elements != null && elements.size() > 0) {
					WinBiddingDTO winBidding = new WinBiddingDTO();
					
					String title = elements.select(".zw_c_cont > .zw_c_c_title").text();
					winBidding.setProjectCode(StringUtils.matchProjectCode(title));
					
					//获取中标人数据
					String winningBidderHtml = elements.select("#generalArticleEditForm00080101_bidOrgDetailTDP").html();
					String winningBidder = StringUtils.matchWinBiddingName(winningBidderHtml);
					//System.out.println(winningBidder);
					winBidding.setWinningBidder(winningBidder);
					
					//获取中标金额
					String winningBidPriceText = elements.select("span").get(5).text();
					winningBidPriceText=winningBidPriceText.replace("中标金额： ", "");
					winningBidPriceText=winningBidPriceText.replace(" 元", "元");
					winBidding.setWinningBidPrice(winningBidPriceText);
//					System.out.println(winningBidPriceText);
					
					//获取发布日期
					String biddingTimeText = elements.select("span").get(2).text();
					biddingTimeText=biddingTimeText.replace("发布日期：", "");
					String pubdate = biddingTimeText.substring(0, 10);
					winBidding.setPubdate(pubdate);
//					System.out.println(pubdate);
					
					//获取中标时间
					String zwcccontText = elements.select(".zw_c_c_cont").text();
					String winBidTime = StringUtils.matchWinBidTime(zwcccontText);
					winBidding.setBiddingTime(winBidTime);
					
					//获取采购项目编号
					String projectCode = elements.select("span").get(3).text();
					projectCode=projectCode.replace("采购项目编号：", "");
					winBidding.setProjectCode(projectCode);
					
					//预算金额
					String budget = "";
					String utilRegex5 = "采购项目预算金额（元）：[\\s\\S]+四、";
					Pattern pattern5 = Pattern.compile(utilRegex5);
					Matcher matcher5 = pattern5.matcher(zwcccontText);
					while (matcher5.find()) {
						budget = matcher5.group();
						budget = budget.replace("采购项目预算金额（元）：", "");
						budget = budget.replace("四、", "");
					}
					if (budget.length()>100) {
						budget="";
					}
					winBidding.setBudget(budget);
					
					//获取客户（发标、采购方）
					String customer = "";
					String utilRegex7 = "受[\\s\\S]+的委托";
					Pattern pattern7 = Pattern.compile(utilRegex7);
					Matcher matcher7 = pattern7.matcher(zwcccontText);
					while (matcher7.find()) {
						customer = matcher7.group();
						customer = customer.replace("受", "");
						customer = customer.replace("的委托", "");
					}
					if (customer.length()>100) {
						customer="";
					}
//					System.out.println("客户（发标、采购方）："+customer);
					winBidding.setCustomer(customer);
					
					//发标时间
					String publishTime = "";
					String utilRegex8 = "于[\\s\\S]+日就";
					Pattern pattern8 = Pattern.compile(utilRegex8);
					Matcher matcher8 = pattern8.matcher(zwcccontText);
					while (matcher8.find()) {
						publishTime = matcher8.group();
						publishTime = publishTime.replace("于", "");
						publishTime = publishTime.replace("就", "");
					}
					if (publishTime.length()>100) {
						publishTime="";
					}
					winBidding.setPublishTime(publishTime);
					
					//公告详情页面正文内容
					Elements contElements = doc.select(".zw_c_c_cont");
					if(contElements != null && contElements.size() > 0) {
						//查找正文中所有p标签下面的span标签
						Elements allPElements = contElements.select("p");
						if(allPElements != null && allPElements.size() > 0){
							for(Element allPElement : allPElements){
								Elements spanElements = allPElement.select("span");
								ListIterator<Element> htmlSpanResultIter = spanElements.listIterator();
								
								while(htmlSpanResultIter.hasNext()) {
									Element get = htmlSpanResultIter.next();
									String text = get.text();
									if(!text.contains("二、采购项目名称：")) {
										continue;
									}
									
									if(text.contains("二、采购项目名称：")) {
										String projectName = "";
										if(text.split("：").length > 1) {
											projectName = text.split("：")[1];
										} else {
											projectName = allPElement.select("span").get(1).text();
										}
										winBidding.setProjectName(projectName.replace("\u00A0", "").replace("；", ""));
									}
									winBidding.setRecordId(downloadBiddingHtmlDTO.getRecordId());
									winBidding.setPurchaseId("1");
									winBidding.setNoticeId("2");
									
									this.bidService.saveWinBidding(winBidding);
								}
							}
						}
					}
				}
			}
			System.out.println("解析完成！");
		}
	}
	
	/**
	 * 验证中标字段不为空
	 * @param bidding
	 * @return
	 */
	private Boolean checkWinBiddingNotNull(WinBiddingDTO winBidding) {
		System.out.println("check==>" + JSONObject.toJSONString(winBidding));
		if (!StringUtils.isNullOrEmpty(winBidding.getWinningBidder()) && !StringUtils.isNullOrEmpty(winBidding.getProjectName())
				&& !StringUtils.isNullOrEmpty(winBidding.getProjectCode()) && !StringUtils.isNullOrEmpty(winBidding.getWinningBidPrice())
				&& !StringUtils.isNullOrEmpty(winBidding.getBiddingTime()) && !StringUtils.isNullOrEmpty(winBidding.getPurchaseId())
				&& !StringUtils.isNullOrEmpty(winBidding.getNoticeId())) {
			return true;
		}
		return false;
	}
	
	/**
	 * 验证更正公告不为空
	 * @param bidding
	 * @return
	 */
	private Boolean checkChangeNoticeNotNull(ChangeNoticeDTO changeNotice) {
		System.out.println("check==>" + JSONObject.toJSONString(changeNotice));
		if (!StringUtils.isNullOrEmpty(changeNotice.getChannelNoticeHtml()) && !StringUtils.isNullOrEmpty(changeNotice.getProjectCode())) {
			return true;
		}
		return false;
	}
	
	/**
	 * 解析更正公告
	 * @return
	 */
	public void resolveChangeNoticeHtml() throws Exception{
		//根据t_download_bidding_html表，查询所有更正公告html
		//定义公告类型
		String noticeType = "更正公告";
		List<DownloadBiddingHtmlDTO> listDownloadBiddingHtmlDTO = bidService.downloadBiddingHtmlByNoticeType(noticeType);
		
		if (listDownloadBiddingHtmlDTO != null && listDownloadBiddingHtmlDTO.size()>0) {
			for (DownloadBiddingHtmlDTO downloadBiddingHtmlDTO : listDownloadBiddingHtmlDTO) {
				String result = downloadBiddingHtmlDTO.getHtmlContent();
				//System.out.println(result);
		        
				Document doc = Jsoup.parse(result);
				Elements elements = doc.select(".zw_contianer");
				if(elements != null && elements.size() > 0) {
					ChangeNoticeDTO changeNotice = new ChangeNoticeDTO();
					
					String title = elements.select(".zw_c_cont > .zw_c_c_title").text();
					changeNotice.setProjectCode(StringUtils.matchProjectCode(title));
					
					//获取正文text
					String NoticeText = elements.select(".zw_c_c_cont").text();
					//System.out.println(NoticeText);
					
					//获取变更内容
					String changeNoticeHtml = StringUtils.matchChangeNoticeContent(NoticeText, "更正/变更：", "其他内容不变。 二、");
					if ( null==changeNoticeHtml || changeNoticeHtml.length()==0) {
						changeNoticeHtml = StringUtils.matchChangeNoticeContent(NoticeText, "澄清/更正：", "其他内容不变。 二、");
					}
					//System.out.println(changeNoticeHtml);
					changeNotice.setChannelNoticeHtml(changeNoticeHtml);
					
					//获取采购项目编号
					String projectCode = elements.select("span").get(3).text();
					projectCode=projectCode.replace("采购项目编号： ", "");
					projectCode=projectCode.replace("采购编号：", "");
					changeNotice.setProjectCode(projectCode);
					
					//保存投标网公告标题ID
					changeNotice.setRecordId(downloadBiddingHtmlDTO.getRecordId());
					
					//插入变更公告对象到t_change_notice表中
					bidService.saveChangeNotice(changeNotice);
				}
			
			}
		}
	}
	
	private String getHtmlContent(String bidUrl) {
		String result = "";
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			if(!Constants.IS_PROXY){
				okHttpResponse = FastHttpClient
						.post()
						.url(bidUrl)
						.build()
						.execute();
			}else{
				Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
				okHttpResponse = FastHttpClient
						.newBuilder()
						.proxy(proxy)
						.build()
						.post()
						.url(bidUrl)
						.build()
						.execute();
			}
			result = okHttpResponse.string();
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			//System.err.println("===连接超时，重新调用getHtmlContent()方法===");
			//sleepTime();
			getHtmlContent(bidUrl);
		} catch (Exception e) {
			e.printStackTrace();
			//System.err.println("===其他异常，重新调用getHtmlContent()方法===");
			//sleepTime();
			getHtmlContent(bidUrl);
		}
		return result;
	}
	
	private String postHtmlContent(String keyword, String channelCode, Integer pageIndex) {
		String result = "";
		String bidUrl = "http://www.gdgpo.gov.cn/queryMoreInfoList.do";
		try {
			Map<String, String> paramMap = new HashMap<String, String>();
			paramMap.put("channelCode", channelCode);
			paramMap.put("sitewebName", "广东省");
			paramMap.put("pageIndex", pageIndex.toString());
			paramMap.put("sitewebId", "4028889705bebb510105bec068b00003");
			paramMap.put("operateDateFrom", "2018-01-01");
			paramMap.put("operateDateTo", "2018-07-26");
			//paramMap.put("title", keyword);
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			if(!Constants.IS_PROXY){
				okHttpResponse = FastHttpClient
						.post()
						.addParams(paramMap)
						.url(bidUrl)
						.build()
						.execute();
			}else{
				Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
				okHttpResponse = FastHttpClient
						.newBuilder()
						.proxy(proxy)
						.build()
						.post()
						.addParams(paramMap)
						.url(bidUrl)
						.build()
						.execute();
			}
			result = okHttpResponse.string();
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用postHtmlContent()方法===");
			postHtmlContent(keyword, channelCode, pageIndex);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用postHtmlContent()方法===");
			postHtmlContent(keyword, channelCode, pageIndex);
		}
		return result;
	}
	
	private void sleepTime() {
		try {
			Thread.sleep(sleepTime);
			System.err.println("休眠" + sleepTime + "毫秒后，往下执行");
		} catch (Exception e){}
	}
	
	private void sleepTime(Long sleepTime) {
		try {
			Thread.sleep(sleepTime);
			System.err.println("休眠" + sleepTime + "毫秒后，往下执行");
		} catch (Exception e){}
	}
	
	public static void main(String[] args) throws Exception{
		BidSpider bidSpider = new BidSpider();
//		bidSpider.biddingNoticeData(1);
//		bidSpider.saveDownloadBiddingHtml("大数据");
		
		//解析采购公告HTML
//		bidSpider.resolveBiddingHtml("000801");
		
		//解析中标公告HTML
		//bidSpider.resolveWinBiddingHtml("000801");
		
		//解析变更公告
		//bidSpider.resolveChangeNoticeHtml();
		
		//System.out.println(bidSpider.postHtmlContent("", "000801", 1));
		String zwcccontText = "七、评审日期：2018-07-24评审地点：广州市越华路118号之一302室";
		String winBidTime = StringUtils.matchWinBidTime(zwcccontText);
		System.out.println(winBidTime+"xx");
	}
}